diff --git a/swh/web/assets/src/bundles/browse/origin-save.js b/swh/web/assets/src/bundles/browse/origin-save.js
index 4df34a04d..f12419dae 100644
--- a/swh/web/assets/src/bundles/browse/origin-save.js
+++ b/swh/web/assets/src/bundles/browse/origin-save.js
@@ -1,252 +1,259 @@
/**
* Copyright (C) 2018 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
import {handleFetchError, csrfPost, isGitRepoUrl, removeUrlFragment} from 'utils/functions';
import {validate} from 'validate.js';
let saveRequestsTable;
function originSaveRequest(originType, originUrl,
acceptedCallback, pendingCallback, errorCallback) {
let addSaveOriginRequestUrl = Urls.browse_origin_save_request(originType, originUrl);
- let grecaptchaData = {'g-recaptcha-response': grecaptcha.getResponse()};
+ let grecaptchaData = {};
+ if (swh.webapp.isReCaptchaActivated()) {
+ grecaptchaData['g-recaptcha-response'] = grecaptcha.getResponse();
+ }
let headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
};
let body = JSON.stringify(grecaptchaData);
csrfPost(addSaveOriginRequestUrl, headers, body)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
if (data.save_request_status === 'accepted') {
acceptedCallback();
} else {
pendingCallback();
}
- grecaptcha.reset();
+ if (swh.webapp.isReCaptchaActivated()) {
+ grecaptcha.reset();
+ }
})
.catch(response => {
if (response.status === 403) {
errorCallback();
}
- grecaptcha.reset();
+ if (swh.webapp.isReCaptchaActivated()) {
+ grecaptcha.reset();
+ }
});
}
export function initOriginSave() {
$(document).ready(() => {
$.fn.dataTable.ext.errMode = 'throw';
fetch(Urls.browse_origin_save_types_list())
.then(response => response.json())
.then(data => {
for (let originType of data) {
$('#swh-input-origin-type').append(``);
}
});
saveRequestsTable = $('#swh-origin-save-requests').DataTable({
serverSide: true,
ajax: Urls.browse_origin_save_requests_list('all'),
columns: [
{
data: 'save_request_date',
name: 'request_date',
render: (data, type, row) => {
if (type === 'display') {
let date = new Date(data);
return date.toLocaleString();
}
return data;
}
},
{
data: 'origin_type',
name: 'origin_type'
},
{
data: 'origin_url',
name: 'origin_url',
render: (data, type, row) => {
if (type === 'display') {
return `${data}`;
}
return data;
}
},
{
data: 'save_request_status',
name: 'status'
},
{
data: 'save_task_status',
name: 'loading_task_status',
render: (data, type, row) => {
if (data === 'succeed') {
let browseOriginUrl = Urls.browse_origin(row.origin_url);
if (row.visit_date) {
browseOriginUrl += `visit/${row.visit_date}/`;
}
return `${data}`;
}
return data;
}
}
],
scrollY: '50vh',
scrollCollapse: true,
order: [[0, 'desc']]
});
$('#swh-origin-save-requests-list-tab').on('shown.bs.tab', () => {
saveRequestsTable.draw();
window.location.hash = '#requests';
});
$('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => {
removeUrlFragment();
});
let saveRequestAcceptedAlert =
`
The "save code now" request has been accepted and will be processed as soon as possible.
`;
let saveRequestPendingAlert =
`
The "save code now" request has been put in pending state and may be accepted for processing after manual review.
`;
let saveRequestRejectedAlert =
`
The "save code now" request has been rejected because the reCAPTCHA could not be validated or the provided origin url is blacklisted.
`;
$('#swh-save-origin-form').submit(event => {
event.preventDefault();
event.stopPropagation();
$('.alert').alert('close');
if (event.target.checkValidity()) {
$(event.target).removeClass('was-validated');
let originType = $('#swh-input-origin-type').val();
let originUrl = $('#swh-input-origin-url').val();
originSaveRequest(originType, originUrl,
() => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert),
() => $('#swh-origin-save-request-status').html(saveRequestPendingAlert),
() => {
$('#swh-origin-save-request-status').css('color', 'red');
$('#swh-origin-save-request-status').html(saveRequestRejectedAlert);
});
} else {
$(event.target).addClass('was-validated');
}
});
$('#swh-show-origin-save-requests-list').on('click', (event) => {
event.preventDefault();
$('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show');
});
$('#swh-input-origin-url').on('input', function(event) {
let originUrl = $(this).val().trim();
$(this).val(originUrl);
$('#swh-input-origin-type option').each(function() {
let val = $(this).val();
if (val && originUrl.includes(val)) {
$(this).prop('selected', true);
}
});
});
if (window.location.hash === '#requests') {
$('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show');
}
});
}
export function validateSaveOriginUrl(input) {
let validUrl = validate({website: input.value}, {
website: {
url: {
schemes: ['http', 'https', 'svn', 'git']
}
}
}) === undefined;
let originType = $('#swh-input-origin-type').val();
if (originType === 'git' && validUrl) {
// additional checks for well known code hosting providers
let githubIdx = input.value.indexOf('://github.com');
let gitlabIdx = input.value.indexOf('://gitlab.');
let gitSfIdx = input.value.indexOf('://git.code.sf.net');
let bitbucketIdx = input.value.indexOf('://bitbucket.org');
if (githubIdx !== -1 && githubIdx <= 5) {
validUrl = isGitRepoUrl(input.value, 'github.com');
} else if (gitlabIdx !== -1 && gitlabIdx <= 5) {
let startIdx = gitlabIdx + 3;
let idx = input.value.indexOf('/', startIdx);
if (idx !== -1) {
let gitlabDomain = input.value.substr(startIdx, idx - startIdx);
// GitLab repo url needs to be suffixed by '.git' in order to be successfully loaded
validUrl = isGitRepoUrl(input.value, gitlabDomain) && input.value.endsWith('.git');
} else {
validUrl = false;
}
} else if (gitSfIdx !== -1 && gitSfIdx <= 5) {
validUrl = isGitRepoUrl(input.value, 'git.code.sf.net/p');
} else if (bitbucketIdx !== -1 && bitbucketIdx <= 5) {
validUrl = isGitRepoUrl(input.value, 'bitbucket.org');
}
}
if (validUrl) {
input.setCustomValidity('');
} else {
input.setCustomValidity('The origin url is not valid or does not reference a code repository');
}
}
export function initTakeNewSnapshot() {
let newSnapshotRequestAcceptedAlert =
`
The "take new snapshot" request has been accepted and will be processed as soon as possible.
`;
let newSnapshotRequestPendingAlert =
`
The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.
`;
let newSnapshotRequestRejectedAlert =
`
The "take new snapshot" request has been rejected because the reCAPTCHA could not be validated.
`;
$(document).ready(() => {
$('#swh-take-new-snapshot-form').submit(event => {
event.preventDefault();
event.stopPropagation();
let originType = $('#swh-input-origin-type').val();
let originUrl = $('#swh-input-origin-url').val();
originSaveRequest(originType, originUrl,
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert),
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert),
() => {
$('#swh-take-new-snapshot-request-status').css('color', 'red');
$('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRejectedAlert);
});
});
});
}
diff --git a/swh/web/assets/src/bundles/webapp/webapp-utils.js b/swh/web/assets/src/bundles/webapp/webapp-utils.js
index 8705d35f9..28c2c5a3a 100644
--- a/swh/web/assets/src/bundles/webapp/webapp-utils.js
+++ b/swh/web/assets/src/bundles/webapp/webapp-utils.js
@@ -1,139 +1,149 @@
import objectFitImages from 'object-fit-images';
import {Layout} from 'admin-lte';
let collapseSidebar = false;
let previousSidebarState = localStorage.getItem('swh-sidebar-collapsed');
if (previousSidebarState !== undefined) {
collapseSidebar = JSON.parse(previousSidebarState);
}
// adapt implementation of fixLayoutHeight from admin-lte
Layout.prototype.fixLayoutHeight = () => {
let heights = {
window: $(window).height(),
header: $('.main-header').outerHeight(),
footer: $('.footer').outerHeight(),
sidebar: $('.main-sidebar').height(),
topbar: $('.swh-top-bar').height()
};
let offset = 10;
$('.content-wrapper').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset);
$('.main-sidebar').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset);
};
$(document).on('DOMContentLoaded', () => {
// restore previous sidebar state (collapsed/expanded)
if (collapseSidebar) {
// hack to avoid animated transition for collapsing sidebar
// when loading a page
let sidebarTransition = $('.main-sidebar, .main-sidebar:before').css('transition');
let sidebarEltsTransition = $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition');
$('.main-sidebar, .main-sidebar:before').css('transition', 'none');
$('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', 'none');
$('body').addClass('sidebar-collapse');
$('.swh-words-logo-swh').css('visibility', 'visible');
// restore transitions for user navigation
setTimeout(() => {
$('.main-sidebar, .main-sidebar:before').css('transition', sidebarTransition);
$('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', sidebarEltsTransition);
});
}
});
$(document).on('collapsed.lte.pushmenu', event => {
if ($('body').width() > 980) {
$('.swh-words-logo-swh').css('visibility', 'visible');
}
});
$(document).on('shown.lte.pushmenu', event => {
$('.swh-words-logo-swh').css('visibility', 'hidden');
});
function ensureNoFooterOverflow() {
$('body').css('padding-bottom', $('footer').outerHeight() + 'px');
}
$(document).ready(() => {
// redirect to last browse page if any when clicking on the 'Browse' entry
// in the sidebar
$(`.swh-browse-link`).click(event => {
let lastBrowsePage = sessionStorage.getItem('last-browse-page');
if (lastBrowsePage) {
event.preventDefault();
window.location = lastBrowsePage;
}
});
// ensure footer do not overflow main content for mobile devices
// or after resizing the browser window
ensureNoFooterOverflow();
$(window).resize(function() {
ensureNoFooterOverflow();
if ($('body').hasClass('sidebar-collapse') && $('body').width() > 980) {
$('.swh-words-logo-swh').css('visibility', 'visible');
}
});
// activate css polyfill 'object-fit: contain' in old browsers
objectFitImages();
// reparent the modals to the top navigation div in order to be able
// to display them
$('.swh-browse-top-navigation').append($('.modal'));
});
export function initPage(page) {
$(document).ready(() => {
// set relevant sidebar link to page active
$(`.swh-${page}-item`).addClass('active');
$(`.swh-${page}-link`).addClass('active');
// triggered when unloading the current page
$(window).on('unload', () => {
// backup sidebar state (collapsed/expanded)
let sidebarCollapsed = $('body').hasClass('sidebar-collapse');
localStorage.setItem('swh-sidebar-collapsed', JSON.stringify(sidebarCollapsed));
// backup current browse page
if (page === 'browse') {
sessionStorage.setItem('last-browse-page', window.location);
}
});
});
}
export function showModalMessage(title, message) {
$('#swh-web-modal-message .modal-title').text(title);
$('#swh-web-modal-message .modal-content p').text(message);
$('#swh-web-modal-message').modal('show');
}
export function showModalConfirm(title, message, callback) {
$('#swh-web-modal-confirm .modal-title').text(title);
$('#swh-web-modal-confirm .modal-content p').text(message);
$('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').bind('click', () => {
callback();
$('#swh-web-modal-confirm').modal('hide');
$('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').unbind('click');
});
$('#swh-web-modal-confirm').modal('show');
}
let swhObjectIcons;
export function setSwhObjectIcons(icons) {
swhObjectIcons = icons;
}
export function getSwhObjectIcon(swhObjectType) {
return swhObjectIcons[swhObjectType];
}
export function initTableRowLinks(trSelector) {
$(trSelector).on('click', function() {
window.location = $(this).data('href');
return false;
});
$('td > a').on('click', function(e) { e.stopPropagation(); });
}
+
+let reCaptchaActivated;
+
+export function setReCaptchaActivated(activated) {
+ reCaptchaActivated = activated;
+}
+
+export function isReCaptchaActivated() {
+ return reCaptchaActivated;
+}
diff --git a/swh/web/browse/views/origin_save.py b/swh/web/browse/views/origin_save.py
index 2f70e7135..14b779443 100644
--- a/swh/web/browse/views/origin_save.py
+++ b/swh/web/browse/views/origin_save.py
@@ -1,86 +1,86 @@
# Copyright (C) 2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
from django.core.paginator import Paginator
from django.http import HttpResponse, HttpResponseForbidden
from django.views.decorators.http import require_POST
from swh.web.browse.browseurls import browse_route
from swh.web.common.exc import ForbiddenExc
from swh.web.common.models import SaveOriginRequest
from swh.web.common.utils import is_recaptcha_valid
from swh.web.common.origin_save import (
create_save_origin_request, get_savable_origin_types,
get_save_origin_requests_from_queryset
)
@browse_route(r'origin/save/(?P.+)/url/(?P.+)/',
view_name='browse-origin-save-request')
@require_POST
def _browse_origin_save_request(request, origin_type, origin_url):
body_unicode = request.body.decode('utf-8')
body = json.loads(body_unicode)
- if is_recaptcha_valid(request, body['g-recaptcha-response']):
+ if is_recaptcha_valid(request, body.get('g-recaptcha-response')):
try:
response = json.dumps(create_save_origin_request(origin_type,
origin_url),
separators=(',', ': '))
return HttpResponse(response, content_type='application/json')
except ForbiddenExc as exc:
return HttpResponseForbidden(str(exc))
else:
return HttpResponseForbidden('The reCAPTCHA could not be validated !')
@browse_route(r'origin/save/types/list/',
view_name='browse-origin-save-types-list')
def _browse_origin_save_types_list(request):
origin_types = json.dumps(get_savable_origin_types(),
separators=(',', ': '))
return HttpResponse(origin_types, content_type='application/json')
@browse_route(r'origin/save/requests/list/(?P.+)/',
view_name='browse-origin-save-requests-list')
def _browse_origin_save_requests_list(request, status):
if status != 'all':
save_requests = SaveOriginRequest.objects.filter(status=status)
else:
save_requests = SaveOriginRequest.objects.all()
table_data = {}
table_data['recordsTotal'] = save_requests.count()
table_data['draw'] = int(request.GET['draw'])
search_value = request.GET['search[value]']
column_order = request.GET['order[0][column]']
field_order = request.GET['columns[%s][name]' % column_order]
order_dir = request.GET['order[0][dir]']
if order_dir == 'desc':
field_order = '-' + field_order
save_requests = save_requests.order_by(field_order)
length = int(request.GET['length'])
page = int(request.GET['start']) / length + 1
save_requests = get_save_origin_requests_from_queryset(save_requests)
if search_value:
save_requests = \
[sr for sr in save_requests
if search_value.lower() in sr['save_request_status'].lower()
or search_value.lower() in sr['save_task_status'].lower()
or search_value.lower() in sr['origin_type'].lower()
or search_value.lower() in sr['origin_url'].lower()]
table_data['recordsFiltered'] = len(save_requests)
paginator = Paginator(save_requests, length)
table_data['data'] = paginator.page(page).object_list
table_data_json = json.dumps(table_data, separators=(',', ': '))
return HttpResponse(table_data_json, content_type='application/json')
diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py
index 9362001e8..d43c8f6f3 100644
--- a/swh/web/common/utils.py
+++ b/swh/web/common/utils.py
@@ -1,350 +1,355 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import docutils.parsers.rst
import docutils.utils
import re
import requests
from datetime import datetime, timezone
from dateutil import parser as date_parser
from dateutil import tz
from django.urls import reverse as django_reverse
from django.http import QueryDict
from swh.model.exceptions import ValidationError
from swh.model.identifiers import (
persistent_identifier, parse_persistent_identifier,
CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT
)
from swh.web.common.exc import BadInputExc
from swh.web.config import get_config
swh_object_icons = {
'branch': 'fa fa-code-fork',
'branches': 'fa fa-code-fork',
'content': 'fa fa-file-text',
'directory': 'fa fa-folder',
'person': 'fa fa-user',
'revisions history': 'fa fa-history',
'release': 'fa fa-tag',
'releases': 'fa fa-tag',
'revision': 'octicon octicon-git-commit',
'snapshot': 'fa fa-camera',
'visits': 'fa fa-calendar',
}
def reverse(viewname, url_args=None, query_params=None,
current_app=None, urlconf=None):
"""An override of django reverse function supporting query parameters.
Args:
viewname (str): the name of the django view from which to compute a url
url_args (dict): dictionary of url arguments indexed by their names
query_params (dict): dictionary of query parameters to append to the
reversed url
current_app (str): the name of the django app tighten to the view
urlconf (str): url configuration module
Returns:
str: the url of the requested view with processed arguments and
query parameters
"""
if url_args:
url_args = {k: v for k, v in url_args.items() if v is not None}
url = django_reverse(viewname, urlconf=urlconf, kwargs=url_args,
current_app=current_app)
if query_params:
query_params = {k: v for k, v in query_params.items() if v}
if query_params and len(query_params) > 0:
query_dict = QueryDict('', mutable=True)
for k in sorted(query_params.keys()):
query_dict[k] = query_params[k]
url += ('?' + query_dict.urlencode(safe='/;:'))
return url
def datetime_to_utc(date):
"""Returns datetime in UTC without timezone info
Args:
date (datetime.datetime): input datetime with timezone info
Returns:
datetime.datetime: datetime in UTC without timezone info
"""
if date.tzinfo:
return date.astimezone(tz.gettz('UTC')).replace(tzinfo=timezone.utc)
else:
return date
def parse_timestamp(timestamp):
"""Given a time or timestamp (as string), parse the result as UTC datetime.
Returns:
datetime.datetime: a timezone-aware datetime representing the
parsed value or None if the parsing fails.
Samples:
- 2016-01-12
- 2016-01-12T09:19:12+0100
- Today is January 1, 2047 at 8:21:00AM
- 1452591542
"""
if not timestamp:
return None
try:
date = date_parser.parse(timestamp, ignoretz=False, fuzzy=True)
return datetime_to_utc(date)
except Exception:
try:
return datetime.utcfromtimestamp(float(timestamp)).replace(
tzinfo=timezone.utc)
except (ValueError, OverflowError) as e:
raise BadInputExc(e)
def shorten_path(path):
"""Shorten the given path: for each hash present, only return the first
8 characters followed by an ellipsis"""
sha256_re = r'([0-9a-f]{8})[0-9a-z]{56}'
sha1_re = r'([0-9a-f]{8})[0-9a-f]{32}'
ret = re.sub(sha256_re, r'\1...', path)
return re.sub(sha1_re, r'\1...', ret)
def format_utc_iso_date(iso_date, fmt='%d %B %Y, %H:%M UTC'):
"""Turns a string representation of an ISO 8601 date string
to UTC and format it into a more human readable one.
For instance, from the following input
string: '2017-05-04T13:27:13+02:00' the following one
is returned: '04 May 2017, 11:27 UTC'.
Custom format string may also be provided
as parameter
Args:
iso_date (str): a string representation of an ISO 8601 date
fmt (str): optional date formatting string
Returns:
str: a formatted string representation of the input iso date
"""
if not iso_date:
return iso_date
date = parse_timestamp(iso_date)
return date.strftime(fmt)
def gen_path_info(path):
"""Function to generate path data navigation for use
with a breadcrumb in the swh web ui.
For instance, from a path /folder1/folder2/folder3,
it returns the following list::
[{'name': 'folder1', 'path': 'folder1'},
{'name': 'folder2', 'path': 'folder1/folder2'},
{'name': 'folder3', 'path': 'folder1/folder2/folder3'}]
Args:
path: a filesystem path
Returns:
list: a list of path data for navigation as illustrated above.
"""
path_info = []
if path:
sub_paths = path.strip('/').split('/')
path_from_root = ''
for p in sub_paths:
path_from_root += '/' + p
path_info.append({'name': p,
'path': path_from_root.strip('/')})
return path_info
def get_swh_persistent_id(object_type, object_id, scheme_version=1):
"""
Returns the persistent identifier for a swh object based on:
* the object type
* the object id
* the swh identifiers scheme version
Args:
object_type (str): the swh object type
(content/directory/release/revision/snapshot)
object_id (str): the swh object id (hexadecimal representation
of its hash value)
scheme_version (int): the scheme version of the swh
persistent identifiers
Returns:
str: the swh object persistent identifier
Raises:
BadInputExc: if the provided parameters do not enable to
generate a valid identifier
"""
try:
swh_id = persistent_identifier(object_type, object_id, scheme_version)
except ValidationError as e:
raise BadInputExc('Invalid object (%s) for swh persistent id. %s' %
(object_id, e))
else:
return swh_id
def resolve_swh_persistent_id(swh_id, query_params=None):
"""
Try to resolve a Software Heritage persistent id into an url for
browsing the pointed object.
Args:
swh_id (str): a Software Heritage persistent identifier
query_params (django.http.QueryDict): optional dict filled with
query parameters to append to the browse url
Returns:
dict: a dict with the following keys:
* **swh_id_parsed (swh.model.identifiers.PersistentId)**: the parsed identifier
* **browse_url (str)**: the url for browsing the pointed object
Raises:
BadInputExc: if the provided identifier can not be parsed
""" # noqa
try:
swh_id_parsed = parse_persistent_identifier(swh_id)
object_type = swh_id_parsed.object_type
object_id = swh_id_parsed.object_id
browse_url = None
query_dict = QueryDict('', mutable=True)
if query_params and len(query_params) > 0:
for k in sorted(query_params.keys()):
query_dict[k] = query_params[k]
if 'origin' in swh_id_parsed.metadata:
query_dict['origin'] = swh_id_parsed.metadata['origin']
if object_type == CONTENT:
query_string = 'sha1_git:' + object_id
fragment = ''
if 'lines' in swh_id_parsed.metadata:
lines = swh_id_parsed.metadata['lines'].split('-')
fragment += '#L' + lines[0]
if len(lines) > 1:
fragment += '-L' + lines[1]
browse_url = reverse('browse-content',
url_args={'query_string': query_string},
query_params=query_dict) + fragment
elif object_type == DIRECTORY:
browse_url = reverse('browse-directory',
url_args={'sha1_git': object_id},
query_params=query_dict)
elif object_type == RELEASE:
browse_url = reverse('browse-release',
url_args={'sha1_git': object_id},
query_params=query_dict)
elif object_type == REVISION:
browse_url = reverse('browse-revision',
url_args={'sha1_git': object_id},
query_params=query_dict)
elif object_type == SNAPSHOT:
browse_url = reverse('browse-snapshot',
url_args={'snapshot_id': object_id},
query_params=query_dict)
except ValidationError as ve:
raise BadInputExc('Error when parsing identifier. %s' %
' '.join(ve.messages))
else:
return {'swh_id_parsed': swh_id_parsed,
'browse_url': browse_url}
def parse_rst(text, report_level=2):
"""
Parse a reStructuredText string with docutils.
Args:
text (str): string with reStructuredText markups in it
report_level (int): level of docutils report messages to print
(1 info 2 warning 3 error 4 severe 5 none)
Returns:
docutils.nodes.document: a parsed docutils document
"""
parser = docutils.parsers.rst.Parser()
components = (docutils.parsers.rst.Parser,)
settings = docutils.frontend.OptionParser(
components=components).get_default_values()
settings.report_level = report_level
document = docutils.utils.new_document('rst-doc', settings=settings)
parser.parse(text, document)
return document
def get_client_ip(request):
"""
Return the client IP address from an incoming HTTP request.
Args:
request (django.http.HttpRequest): the incoming HTTP request
Returns:
str: The client IP address
"""
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR')
return ip
def is_recaptcha_valid(request, recaptcha_response):
"""
Verify if the response for Google reCAPTCHA is valid.
Args:
request (django.http.HttpRequest): the incoming HTTP request
recaptcha_response (str): the reCAPTCHA response
Returns:
bool: Whether the reCAPTCHA response is valid or not
"""
config = get_config()
- return requests.post(
- config['grecaptcha']['validation_url'],
- data={
- 'secret': config['grecaptcha']['private_key'],
- 'response': recaptcha_response,
- 'remoteip': get_client_ip(request)
- },
- verify=True
- ).json().get("success", False)
+ if config['grecaptcha']['activated'] is False:
+ recaptcha_valid = True
+ else:
+ recaptcha_valid = requests.post(
+ config['grecaptcha']['validation_url'],
+ data={
+ 'secret': config['grecaptcha']['private_key'],
+ 'response': recaptcha_response,
+ 'remoteip': get_client_ip(request)
+ },
+ verify=True
+ ).json().get("success", False)
+ return recaptcha_valid
def context_processor(request):
"""
Django context processor used to inject variables
in all swh-web templates.
"""
config = get_config()
return {'swh_object_icons': swh_object_icons,
+ 'grecaptcha_activated': config['grecaptcha']['activated'],
'grecaptcha_site_key': config['grecaptcha']['site_key']}
diff --git a/swh/web/config.py b/swh/web/config.py
index 1c1199ae3..b21e1faa3 100644
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -1,130 +1,131 @@
# Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.core import config
from swh.storage import get_storage
from swh.indexer.storage import get_indexer_storage
from swh.vault.api.client import RemoteVaultClient
from swh.scheduler import get_scheduler
DEFAULT_CONFIG = {
'allowed_hosts': ('list', []),
'storage': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://127.0.0.1:5002/',
'timeout': 10,
},
}),
'indexer_storage': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://127.0.0.1:5007/',
'timeout': 1,
}
}),
'vault': ('string', 'http://127.0.0.1:5005/'),
'log_dir': ('string', '/tmp/swh/log'),
'debug': ('bool', False),
'serve_assets': ('bool', False),
'host': ('string', '127.0.0.1'),
'port': ('int', 5004),
'secret_key': ('string', 'development key'),
# do not display code highlighting for content > 1MB
'content_display_max_size': ('int', 1024 * 1024),
'snapshot_content_max_size': ('int', 1000),
'throttling': ('dict', {
'cache_uri': None, # production: memcached as cache (127.0.0.1:11211)
# development: in-memory cache so None
'scopes': {
'swh_api': {
'limiter_rate': {
'default': '120/h'
},
'exempted_networks': ['127.0.0.0/8']
},
'swh_vault_cooking': {
'limiter_rate': {
'default': '120/h',
'GET': '60/m'
},
'exempted_networks': ['127.0.0.0/8']
},
'swh_save_origin': {
'limiter_rate': {
'default': '120/h',
'POST': '10/h'
},
'exempted_networks': ['127.0.0.0/8']
}
}
}),
'scheduler': ('dict', {
'cls': 'remote',
'args': {
'url': 'http://localhost:5008/'
}
}),
'grecaptcha': ('dict', {
+ 'activated': True,
'validation_url': 'https://www.google.com/recaptcha/api/siteverify',
'site_key': '',
'private_key': ''
}),
'production_db': ('string', '/var/lib/swh/web.sqlite3'),
'deposit': ('dict', {
'private_api_url': 'https://deposit.softwareheritage.org/1/private/',
'private_api_user': 'swhworker',
'private_api_password': ''
})
}
swhweb_config = {}
def get_config(config_file='web/web'):
"""Read the configuration file `config_file`, update the app with
parameters (secret_key, conf) and return the parsed configuration as a
dict. If no configuration file is provided, return a default
configuration."""
if not swhweb_config:
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, 'log_dir')
swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
swhweb_config['vault'] = RemoteVaultClient(swhweb_config['vault'])
swhweb_config['indexer_storage'] = \
get_indexer_storage(**swhweb_config['indexer_storage'])
swhweb_config['scheduler'] = get_scheduler(**swhweb_config['scheduler']) # noqa
return swhweb_config
def storage():
"""Return the current application's storage.
"""
return get_config()['storage']
def vault():
"""Return the current application's vault.
"""
return get_config()['vault']
def indexer_storage():
"""Return the current application's indexer storage.
"""
return get_config()['indexer_storage']
def scheduler():
"""Return the current application's scheduler.
"""
return get_config()['scheduler']
diff --git a/swh/web/templates/browse/layout.html b/swh/web/templates/browse/layout.html
index 7966a659f..d922de2d5 100644
--- a/swh/web/templates/browse/layout.html
+++ b/swh/web/templates/browse/layout.html
@@ -1,24 +1,23 @@
{% extends "layout.html" %}
{% comment %}
Copyright (C) 2017-2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load swh_templatetags %}
{% load render_bundle from webpack_loader %}
{% block title %}{{ heading }} – Software Heritage archive {% endblock %}
{% block header %}
{% render_bundle 'browse' %}
{% render_bundle 'vault' %}
-
{% endblock %}
{% block content %}
Beta version
{% block browse-content %}{% endblock %}
{% endblock %}
diff --git a/swh/web/templates/browse/origin-save.html b/swh/web/templates/browse/origin-save.html
index 96c73bb2c..92e233c67 100644
--- a/swh/web/templates/browse/origin-save.html
+++ b/swh/web/templates/browse/origin-save.html
@@ -1,111 +1,127 @@
{% extends "./layout.html" %}
{% comment %}
Copyright (C) 2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
+{% block header %}
+{{ block.super }}
+{% if grecaptcha_activated %}
+
+{% endif %}
+{% endblock %}
+
{% block navbar-content %}
You can contribute to extend the content of the Software Heritage archive by submitting an origin
save request. To do so, fill the required info in the form below:
Origin type: the type of version control system the software origin is using.
Currently, the only supported type is git, for origins using Git.
Soon, the following origin types will also be available to save into the archive:
Origin url: the url of the remote repository for the software origin.
In order to avoid saving errors from Software Heritage, you should provide the clone/checkout url
as given by the provider hosting the software origin. It can easily be found in the
web interface used to browse the software origin. For instance, if you want to save a git
origin into the archive, you should check that the command $ git clone <origin_url>
does not return an error before submitting a request.
Once submitted, your save request can either be:
accepted: a visit to the provided origin will then be scheduled by Software Heritage in order to
load its content into the archive as soon as possible
rejected: the provided origin url is blacklisted and no visit will be scheduled
put in pending state: a manual review will then be performed in order to determine if the
origin can be safely loaded or not into the archive
{% endblock %}
\ No newline at end of file
diff --git a/swh/web/templates/includes/take-new-snapshot.html b/swh/web/templates/includes/take-new-snapshot.html
index 555d5b3c6..73ce79369 100644
--- a/swh/web/templates/includes/take-new-snapshot.html
+++ b/swh/web/templates/includes/take-new-snapshot.html
@@ -1,73 +1,88 @@
{% comment %}
Copyright (C) 2019 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load swh_templatetags %}
{% if snapshot_context and snapshot_context.origin_info and snapshot_context.origin_info.type|origin_type_savable %}
+ {% if grecaptcha_activated %}
+
+ {% endif %}
+
Take a new snapshot of a software origin
If the archived software origin currently browsed is not synchronized with its upstream
version (for instance when new commits have been issued), you can explicitely request Software
Heritage to take a new snapshot of it.
Use the form below to proceed. Once a request has been submitted and accepted, it will be processed as soon as possible.
You can then check its processing state by visiting this dedicated page.
{% endif %}
\ No newline at end of file
diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html
index 172f3f03d..b04b8a4f8 100644
--- a/swh/web/templates/layout.html
+++ b/swh/web/templates/layout.html
@@ -1,186 +1,187 @@
{% comment %}
Copyright (C) 2015-2018 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load static %}
{% load render_bundle from webpack_loader %}
{% load swh_templatetags %}
{% block title %}{% endblock %}
{% render_bundle 'vendors' %}
{% render_bundle 'webapp' %}
{% block header %}{% endblock %}